package categoryModel;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.StringTokenizer;
import java.util.TreeMap;

import modules.*;
import parameter.*;
import functions.*;


// Input : NaverTFIDF
// Output : WikiIDF

public class GenWikiICFMain {

	public static void main(String args[]) throws Exception{
		
		System.out.println("Start");
		List<File> fileListWIKI = FileFunction.getListOfFiles(Path.CATEGORY_MODEL_WIKI_CF_PATH);
		
    	int totalCategory = 0;
        Map<String, Double> termCounts = new HashMap<String, Double>();
        Map<String, Double> idf = new HashMap<String, Double>();

		for (File file : fileListWIKI) {
			totalCategory++;
			FileReader fr = new FileReader(file);
			BufferedReader br = new BufferedReader(fr);
			String line = "";
			while ((line = br.readLine()) != null) {
				StringTokenizer token = new StringTokenizer(line, "\t");
				if(token.countTokens() >= 2){
					String key= token.nextToken();
					String value= token.nextToken();
					if(termCounts.containsKey(key))
	        			termCounts.put(key, termCounts.get(key) + 1.0  );
	        		else
	        			termCounts.put(key, 1.0);
				}
			}
			br.close();
			fr.close();

		}
		
	    for(Map.Entry<String, Double> termCount : termCounts.entrySet()){
        	idf.put(termCount.getKey(), Math.log( totalCategory / (double)termCount.getValue() )); 	
        }
        
		DoubleValueComparator bvc = new DoubleValueComparator(idf);
		TreeMap<String, Double> tMap = new TreeMap<String, Double>(bvc);
		tMap.putAll(idf);
		
		FileWriter fw = new FileWriter(Path.WIKI_ICF_FILEPATH);
		BufferedWriter bw = new BufferedWriter(fw);
		
    	for(Map.Entry<String, Double> term : tMap.entrySet() ){
    		bw.write ( term.getKey() + "\t" + term.getValue() );
    		bw.newLine();
    	}
        bw.close();
        fw.close();
		System.out.println("Complete");

    	
	}
	
}
